#include <fstream>
#include <iostream>
#include <stdio.h>
#include <string.h>
#include "Parser.h"
#include <cstring>
#include <dirent.h>
#include <algorithm>
#include <vector>
#include <locale>
#include "Extra-Functions.h"
#include <stdlib.h>
using namespace std;


int main() {

	// We create the parser
	Parser *parser = new Parser();

	// We obtain the stopwords from a file and put them in an array
    ofstream myfile2("../Stopwords/english2");
	string stopWords[143];
	parser->setFile("../Stopwords/english");
	int i = 0;
    locale loc;
	while ( parser->hasNextWord() ) {
      	stopWords[i] = parser->nextWord();
      	i++;
	}
	parser->closeFile();

//	if ( isStopWord ( stopWords , 223 , "they") ) { cout << "was está" << endl; }

	long int wordCounter = 0;
    long int bookCounter = 0;
    int bookId = 0;
    int counter = 0;

    // And we create some variables for opening the directory containing the books
    DIR *pDIR = opendir ("../prueba");
    struct dirent *entry;

//    ofstream myfile("/media/3CFD6AA408B383E2/superArchivo2");
    ofstream myfile("../superArchivo2");
    string currentFile,currentWord,currentWordLowered;
    // No more variable declarations from now on

    cout << "Opening directory...";

    if ( pDIR ){

    	cout << " OK" << endl << "Reading files...";

    	// Here we read each book from the given directory
    	while ( entry = readdir(pDIR) ){

    		// Some filters...
    		if( strcmp(entry->d_name, ".") != 0 && strcmp(entry->d_name, "..") != 0 && strcmp(entry->d_name, ".svn") != 0) {

                bookCounter++;

                // Setting the filename
    			string str(entry->d_name);
    			currentFile = "../prueba/" + str;
    			parser->setFile(string(currentFile));
    			counter++;

    			while ( parser->hasNextWord() ) {
    				currentWord = parser->nextWord();
    				currentWordLowered = "";

    				// Just to lowercase the words
    				for (string::size_type i=0; i<currentWord.length(); ++i) currentWordLowered += tolower(currentWord[i],loc);

    				if ( !binary_search ( stopWords , stopWords + 143 , currentWordLowered) ) { wordCounter++; myfile << currentWord  << " " << counter << endl; }

    			}

                parser->closeFile();
    		}
    	}

    	cout << " OK" << endl;
    	closedir(pDIR);
    }

    cout << "Sorting main file...";

    system("cd ../; sort superArchivo2 -o superArchivo2 ");

    cout << " OK" << endl;

    cout << "Total processed words: " << wordCounter << endl;
    cout << "Total processed books: " << bookCounter << endl;

    delete parser;
	return 0;

}
